package spider;

import savantConn.mysql;
import java.io.IOException;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


/**
 * java爬取云代理高匿（http://www.ip3366.net）
 */
public class proxySpider {

  public static Connection mConnect;
  public static Statement statement;

  proxySpider() throws SQLException {

    mConnect = mysql.getConnection();
    statement = mConnect.createStatement();
  }


  public static void main(String[] args) throws Exception {
    proxySpider ps = new proxySpider();
    for (int page = 1; page <= 10; page++) {
      ps.paging(page);
      System.out.println("第一" + page + "页");
      Thread.sleep(500);
    }

  }

  // 分页查询
  public static void paging(int page) throws IOException, SQLException {

    String url = "http://www.ip3366.net/?stype=1&page=" + page;
    String html = getIndex(url);
    String ipInfo = getIpInfo(html);

  }

  // 获取ip详细信息
  public static String getIpInfo(String html) throws SQLException {
    Document doc = Jsoup.parse(html);
    Elements trs = doc.select("#list > table > tbody > tr");

    for (Element element : trs) {
      String info = "";
      Elements tds = element.select("td");

      // 获取代理IP地址
      String ip = tds.get(0).text();
      // 获取端口号
      String port = tds.get(1).text();
      // 获取匿名度
      String anonym = tds.get(2).text();
      // 获取类型
      String type = tds.get(3).text();
      // 获取get/post支持
      String support = tds.get(4).text();
      //获取位置
      String address = tds.get(5).text();
      //获取响应速度
      String speed = tds.get(6).text();
      //获取最后验证时间
      String time = tds.get(7).text();

      String sql =
          "INSERT INTO `proxy`(`ip`, `port`, `anonym`, `type`, `support`, `address`, `speed`, `time`) VALUES ('"
              + ip + "','" + port + "','" + anonym + "','" + type + "','" + support + "','"
              + address + "','" + speed + "','" + time + "')";
      statement.execute(sql);//执行语句

      info += ip;
      info += port;
      info += anonym;
      info += type;
      info += support;
      info += address;
      info += speed;
      info += time;

      System.out.println(info);
    }

    return null;
  }

  // 获取网页
  public static String getIndex(String url) throws IOException {
    //创建客户端
    CloseableHttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet(url);
//         setConnectTimeout(10000)连接超时时间（单位豪秒）
//         setSocketTimeout(10000)读取超时时间（单位豪秒）
    RequestConfig config = RequestConfig.custom().setConnectTimeout(2000)
        .setSocketTimeout(3000).build();
    httpGet.setConfig(config);
    httpGet.setHeader(
        "User-Agent",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36");

    // 执行
    CloseableHttpResponse res = httpClient.execute(httpGet);
    HttpEntity entity = res.getEntity();
    String html = "";
    if (entity != null) {
      html = EntityUtils.toString(entity, Charset.forName("gb2312"));
    }

    return html;
  }

}